package de.dopichaj.labrador.inex;


import java.io.BufferedReader;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.FileReader;
import java.io.IOException;
import java.io.InputStreamReader;
import java.io.OutputStreamWriter;
import java.io.Reader;
import java.io.Writer;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Comparator;
import java.util.List;
import java.util.Map;
import java.util.zip.ZipEntry;
import java.util.zip.ZipOutputStream;

import javax.script.ScriptEngine;
import javax.script.ScriptEngineManager;
import javax.script.ScriptException;
import javax.xml.parsers.ParserConfigurationException;
import javax.xml.parsers.SAXParser;
import javax.xml.parsers.SAXParserFactory;

import org.apache.commons.cli.BasicParser;
import org.apache.commons.cli.CommandLine;
import org.apache.commons.cli.CommandLineParser;
import org.apache.commons.cli.HelpFormatter;
import org.apache.commons.cli.Options;
import org.apache.commons.cli.ParseException;
import org.apache.commons.lang.time.StopWatch;
import org.apache.log4j.ConsoleAppender;
import org.apache.log4j.Level;
import org.apache.log4j.Logger;
import org.apache.log4j.SimpleLayout;
import org.apache.xml.resolver.tools.CatalogResolver;
import org.xml.sax.InputSource;
import org.xml.sax.SAXException;
import org.xml.sax.XMLReader;

import de.dopichaj.labrador.index.Index;
import de.dopichaj.labrador.index.IndexVersionException;
import de.dopichaj.labrador.index.lucene.LuceneFileSystemIndex;
import de.dopichaj.labrador.index.own.OwnFileSystemIndex;
import de.dopichaj.labrador.inex.IncrementalPostProcessor.HitListSorter;
import de.dopichaj.labrador.inex.SubmissionWriter.Task;
import de.dopichaj.labrador.inex.assess.XCGAssessor;
import de.dopichaj.labrador.search.NEXISearcher;
import de.dopichaj.labrador.search.QueryParseException;
import de.dopichaj.labrador.search.SearchException;
import de.dopichaj.labrador.search.erg.ErgFileFormatException;
import de.dopichaj.labrador.search.hit.DocHitMap;
import de.dopichaj.labrador.search.hit.Hit;
import de.dopichaj.labrador.search.prune.BestInContextPruner;
import de.dopichaj.labrador.search.prune.FuzzyLengthBasedPruner;
import de.dopichaj.labrador.search.prune.MultiPruner;
import de.dopichaj.labrador.search.prune.ResultTreePruner;
import de.dopichaj.labrador.search.prune.RootRemovingPruner;
import de.dopichaj.labrador.search.prune.ScoreBasedPruner;
import de.dopichaj.labrador.util.AbortingExceptionHandler;


public class Search {
    
    private static final Logger log = Logger.getLogger(Search.class);
    /**
     * The maximum number of results to include in the submission.
     */
    private static final int MAX_RESULTS = 1500;
    
    private final boolean searchCAS;
    private final int year;
    private final boolean doBestInContext;
    private final String idAppendix;
    private final boolean onlyThorough;
    private final boolean allResults;
    private final boolean zip;
    private final boolean dontWrite;
    private final XMLReader xmlReader;
    private final NEXISearcher searcher;
    private final List<IncrementalPostProcessor> postprocessors;
    private final Map<Integer, XCGAssessor> assessorMap;
    
    public Search(boolean searchCAS, final Index index, int year,
        boolean doBestInContext, String idAppendix,
        boolean onlyThorough, boolean allResults, boolean zip, boolean dontWrite, File patternsFile,
        Map<Integer, XCGAssessor> assessorMap)
        throws IOException, IndexVersionException, ErgFileFormatException, ScriptException {
        
        this.searchCAS = searchCAS;
        this.year = year;
        this.doBestInContext = doBestInContext;
        this.idAppendix = idAppendix;
        this.onlyThorough = onlyThorough;
        this.allResults = allResults;
        this.zip = zip;
        this.dontWrite = dontWrite;
        this.assessorMap = assessorMap;
        
        xmlReader = setupXmlParser();

        searcher = index.getNEXISearcher();

        // set up postprocessors
        if (patternsFile != null) {
            postprocessors = setupPostprocessors(patternsFile, searcher.getDescription());
        } else {
            postprocessors = Collections.emptyList();
        }
    }
    
    public void close() {
        for (final IncrementalPostProcessor pp : postprocessors) {
            pp.close();
        }
        searcher.close();
    }
    
    /**
     * @param commandLine
     * @param searchCas
     * @param postprocessors
     * @param xmlReader
     * @param searcher
     * @throws IOException 
     * @throws SAXException 
     * @throws QueryParseException 
     * @throws SearchException 
     * @throws  
     */
    public void processTopic(final String name) throws IOException,
        SAXException, SearchException, QueryParseException {
        
        log.info("Topic file " + name);
        FileInputStream is = new FileInputStream(name);
        Reader reader = new BufferedReader(new InputStreamReader(is));
        InputSource inputSource = new InputSource(reader);
        Topic topic = new Topic();
        topic.setSearcher(searcher);
        xmlReader.setContentHandler(topic);
        xmlReader.parse(inputSource);

        if (assessorMap == null || assessorMap.containsKey(topic.getId())) {

            log.info("Processing topic " + topic.getId());
            processTopic(topic);
            log.info("Finished processing topic " + topic.getId());
        } else {
            log.info("Skipping topic " + topic.getId() + " (no assessments available)");
        }
    }

    private List<IncrementalPostProcessor>
        setupPostprocessors(File patternsFile, String description) throws IOException, ScriptException {
        
        final List<IncrementalPostProcessor> postprocessors =
            new ArrayList<IncrementalPostProcessor>();
        
        final BufferedReader reader = new BufferedReader(new InputStreamReader(
            new FileInputStream(patternsFile)));

        final String type = searchCAS ? "CAS" : "CO";
        String line;
        while ((line = reader.readLine()) != null) {
            final String[] parts = line.split(":");
            if (parts.length != 4) {
                System.err.println("Malformed pattern file: " + line);
                System.exit(-1);
            }
            final String scriptFile = parts[0];
            final String baseName = parts[1];
            final String fullDescription = description + parts[2];
            final boolean doLengthBasedPruning = parts[3].equals("L");
            final String runId = baseName + parts[3] + idAppendix;
            postprocessors.add(createPostProcessor(runId,
                type, scriptFile, doLengthBasedPruning, fullDescription));
        }
        
        return postprocessors;
    }

    public IncrementalPostProcessor createPostProcessor(String runId,
        String coOrCas, String scriptFile, boolean doLengthBasedPruning,
        String description)
        throws IOException, ScriptException {

        final boolean vvcas = coOrCas.equals("VVCAS");
        IncrementalPostProcessor postProcessor =
            new IncrementalPostProcessor(year, MAX_RESULTS, !allResults, assessorMap);
        
        try {
            String outputfile = coOrCas + "-" + runId;
            final ResultTreePruner defaultPruner;
            defaultPruner = getMultiPatternPruner(scriptFile, doLengthBasedPruning);
            
            // INEX 2007 got rid of "thorough"
            if (year != 2007) {
                final Writer thoroughw = getZipWriter(outputfile + "-thorough.xml");
                postProcessor.addPostProcessor(defaultPruner,
                    thoroughw, coOrCas, Task.THOROUGH, runId, description);
            }

            if (!onlyThorough && !vvcas && !doBestInContext) {
                Writer focusedw = getZipWriter(outputfile + "-focused.xml");
                postProcessor.addPostProcessor(new ScoreBasedPruner(),
                    focusedw, coOrCas, Task.FOCUSED, runId,
                    description);
                Writer ricw = getZipWriter(outputfile + "-ric.xml");
                postProcessor.addPostProcessor(new RootRemovingPruner(),
                    ricw, coOrCas, Task.ALL_IN_CONTEXT, runId,
                    description, new RelevantInContextSorter());
            }
            if (doBestInContext) {
                Writer bicw = getZipWriter(outputfile + "-bic.xml");
                postProcessor.addPostProcessor(new BestInContextPruner(),
                    bicw, coOrCas, Task.BEST_IN_CONTEXT, runId,
                    description);
            }

            return postProcessor;

        } catch (FileNotFoundException e) {
            postProcessor.close();
            throw e;
        }
    }

    private static ResultTreePruner getMultiPatternPruner(String scriptFile,
        boolean doLengthBasedPruning)
        throws FileNotFoundException, IOException, ScriptException {

        ScriptEngineManager factory = new ScriptEngineManager();
        ScriptEngine engine = factory.getEngineByName("JavaScript");
        
        final ResultTreePruner pruner =
            (ResultTreePruner)engine.eval(new FileReader(scriptFile));

        if (doLengthBasedPruning) {
            return new MultiPruner(pruner, new FuzzyLengthBasedPruner());
        } else {
            return pruner;
        }
    }
    
    /**
     * A ZipOutputStream that only contains a single file.
     */
    private static final class OneEntryZipOutputStream extends ZipOutputStream {

        public OneEntryZipOutputStream(String name) throws IOException {
            super(new FileOutputStream(name + ".zip"));
            putNextEntry(new ZipEntry(name));
        }

        @Override
        public void close() throws IOException {
            closeEntry();
            super.close();
        }
    }

    private Writer getZipWriter(String filename) throws IOException {
    
        if (dontWrite) {
            return new Writer() {
                public void close() throws IOException {}
                public void flush() throws IOException {}
                public void write(char[] cbuf, int off, int len) throws IOException {}};
        } else if (zip) {
            return new OutputStreamWriter(new OneEntryZipOutputStream(filename));
        } else {
            return new OutputStreamWriter(new FileOutputStream(filename));
        }
    }

    private static final class RelevantInContextSorter
        implements HitListSorter {
    
        private final Comparator<Hit> defaultComp =
            new LengthScoreComparator();
        
        public List<Hit> sort(List<Hit> hits) {
            
            // keep only the best results
            Collections.sort(hits, defaultComp);
            final List<Hit> newHits = hits.subList(0,
                Math.min(MAX_RESULTS, hits.size()));
            
            // sort by document
            Comparator<Hit> comparator =
                new RelevantInContextComparator(newHits);
            Collections.sort(newHits, comparator);
            
            return newHits;
        }
    }

    private void processTopic(Topic topic)
        throws SearchException, QueryParseException {
        
        StopWatch watch = new StopWatch();
        watch.start();
        
        log.info("  Searching for «" +
            (searchCAS ? topic.getCASTitle() : topic.getTitle()) + "»");
            
        for (IncrementalPostProcessor pp : postprocessors) {
            final DocHitMap results = searchCAS
                ? topic.searchCAS()
                : topic.searchCO();
            if (results != null) {
                pp.pruneAndWrite(topic.getId(), results);
            }
        }
        watch.stop();
        log.info("  searching took " + watch.getTime() + "ms");
    }

    private static XMLReader setupXmlParser() {
        XMLReader xmlReader = null;
        try {
            SAXParserFactory factory = SAXParserFactory.newInstance();
            SAXParser saxParser = factory.newSAXParser();
            xmlReader = saxParser.getXMLReader();
            xmlReader.setEntityResolver(new CatalogResolver());
        } catch (SAXException e) {
            System.err.println("Error creating XML parser: ");
            e.printStackTrace();
            System.exit(1);
        } catch (ParserConfigurationException e) {
            System.err.println("Error creating XML parser: ");
            e.printStackTrace();
            System.exit(1);
        }
        return xmlReader;
    }
    
    
    // -------------------------------------------------------------------------

    private static CommandLine parseCommandLine(String[] args) {
        CommandLineParser parser = new BasicParser();
        Options options = new Options();
        options.addOption("i", "de.dopichaj.labrador.index", true, "The de.dopichaj.labrador.index to use for searching");
        options.addOption("s", "cas", false, "Do CAS de.dopichaj.labrador.search");
        options.addOption("h", "help", false, "Print this help");
        options.addOption("y", "year", true,
            "The year to use for the submission format");
        options.addOption("b", "best-in-context", false, "Do Best in Context " +
            "instead of Focused and All in Context");
        options.addOption("p", "patterns", true, "Use this patterns file");
        options.addOption("k", "keep-docs", true, "The number of documents to " +
            "keep (defaults to maximum integer)");
        options.addOption("a", "append-to-id", true, "A string to append to " +
            "the run IDs");
        options.addOption("t", "only-thorough", false,
            "only do thorough (no focused or all in context)");
        options.addOption("z", "zip", false, "make a zip file of the results");
        options.addOption("W", "dont-write", false, "don't write the results at all");
        options.addOption("A", "all", false, "write all results in DocHitMap format");
        options.addOption("L", "lucene", false, "use lucene for searching");
        options.addOption("C", "configuration", true, "use the given configuration properties file");
        options.addOption("r", "recall-base", true, "use the given recall base");
        CommandLine commandLine = null;
        try {
            commandLine = parser.parse(options, args);
        } catch (ParseException e) {
            System.err.println("Error parsing command line: " + e.getMessage());
            printHelp(options);
            System.exit(-1);
        }
        if (commandLine.hasOption("h")) {
            printHelp(options);
            System.exit(0);
        }
        return commandLine;
    }
    
    private static void printHelp(Options options) {
        HelpFormatter helpFormatter = new HelpFormatter();
        helpFormatter.printHelp(
            "Usage: de.dopichaj.labrador.inex.Search [-i indexdir] [-y year] [-p patterns-file] " +
            "[-r results] [-e] [-t] [-a append] [-z] [-A] [-s] topic ...",
            options);
    }

    public static void main(String[] args) throws ErgFileFormatException,
    IndexVersionException, FileNotFoundException, IOException, ScriptException, SAXException {

        Thread.setDefaultUncaughtExceptionHandler(new AbortingExceptionHandler());
    
        final ConsoleAppender appender =
            new ConsoleAppender(new SimpleLayout(), ConsoleAppender.SYSTEM_ERR);
        appender.setThreshold(Level.INFO);
        log.setLevel(Level.INFO);
        log.addAppender(appender);
    
        // parse the command line options
        final CommandLine commandLine = parseCommandLine(args);
        final boolean searchCAS = commandLine.hasOption("s");
        final File indexDir =
            new File(commandLine.getOptionValue("i", "de.dopichaj.labrador.index"));
        final int year =
            Integer.parseInt(commandLine.getOptionValue("y", "2007"));
        final boolean doBestInContext = commandLine.hasOption("b");
        final String idAppendix = commandLine.getOptionValue('a', "");
        final boolean onlyThorough = commandLine.hasOption('t');
        final boolean allResults = commandLine.hasOption('A');
        final boolean zip = commandLine.hasOption('z');
        final boolean dontWrite = commandLine.hasOption('W');
        final File patternsFile;
        if (commandLine.hasOption("p")) {
            patternsFile = new File(commandLine.getOptionValue("p"));
        } else {
            patternsFile = null;
        }
    
        final Index index;
        if (commandLine.hasOption('L')) {
            index = new LuceneFileSystemIndex(indexDir);
        } else {
            index = new OwnFileSystemIndex(indexDir);
        }
        if (commandLine.hasOption('C')) {
            index.getConfiguration().load(new File(commandLine.getOptionValue('C')));
        }
        final Map<Integer, XCGAssessor> assessorMap;
        if (commandLine.hasOption('r')) {
            Map<Integer, Assessment> assessmentMap = Assessment.readAssessments(new FileReader(commandLine.getOptionValue('r')));
            assessorMap = XCGAssessor.getAssessors(assessmentMap);
        } else {
            assessorMap = null;
        }
        final Search search = new Search(searchCAS, index, year,
            doBestInContext, idAppendix, onlyThorough, allResults, zip, dontWrite,
            patternsFile, assessorMap);
        log.info("Searching " + indexDir);
    
        for (final String name : commandLine.getArgs()) {
            try {
                search.processTopic(name);
            } catch (IOException e) {
                log.error("Error reading file " + name + ": ", e);
            } catch (SAXException e) {
                log.error("Error parsing file " + name + ": ", e);
            } catch (SearchException e) {
                log.error("Search error", e);
            } catch (QueryParseException e) {
                log.error("Error parsing query", e);
            }
        }
    
        search.close();
    }
}
/*
Copyright (c) 2007 Philipp Dopichaj

Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:

The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.

THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
THE SOFTWARE.
*/